#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2025/1/25 16:52
# @Author  : Ramsey
# @Site    : zh
# @File    : baidu_homepage.py
# @Software: PyCharm
from spider_framework.spider import Spider
from spider_framework.http.request import Request
from items import BaiduItem


class BaiduHomePageSpider(Spider):
    start_urls = ["https://www.baidu.com", "https://www.baidu.com"]

    # custom_settings = {
    #     "CONCURRENCY": 3
    # }

    def parse(self, response):
        print(f"parse: {response}")
        for _ in range(2):
            url = "http://www.baidu.com"
            yield Request(url=url, callback=self.parse_page)

    def parse_page(self, response):
        print("parse_page: ", response)
        for _ in range(3):
            url = "http://www.baidu.com"
            yield Request(url=url, callback=self.parse_detail)

    def parse_detail(self, response):
        item = BaiduItem()
        item["url"] = response.url
        item["title"] = response.xpath("//title/text()").get()
        yield item


if __name__ == "__main__":
    pass
